def flatten_list(l):
return [x for y in l for x in y]
def is_acceptable(word: str):
return word not in stop_words and len(word) > 2
# Color coding our wordclouds
def red_color_func(word, font_size, position, orientation, random_state=None,**kwargs):
return f"hsl(0, 100%, {random.randint(25, 75)}%)"
def green_color_func(word, font_size, position, orientation, random_state=None,**kwargs):
return f"hsl({random.randint(90, 150)}, 100%, 30%)"
def yellow_color_func(word, font_size, position, orientation, random_state=None,**kwargs):
return f"hsl(42, 100%, {random.randint(25, 50)}%)"
# Reusable function to generate word clouds
def generate_word_clouds(neg_doc, neu_doc, pos_doc):
# Display the generated image:
fig, axes = plt.subplots(1,3, figsize=(20,10))
wordcloud_neg = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(" ".join(neg_doc))
axes[0].imshow(wordcloud_neg.recolor(color_func=red_color_func, random_state=3), interpolation='bilinear')
axes[0].set_title("Negative Words")
axes[0].axis("off")
wordcloud_neu = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(" ".join(neu_doc))
axes[1].imshow(wordcloud_neu.recolor(color_func=yellow_color_func, random_state=3), interpolation='bilinear')
axes[1].set_title("Neutral Words")
axes[1].axis("off")
wordcloud_pos = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(" ".join(pos_doc))
axes[2].imshow(wordcloud_pos.recolor(color_func=green_color_func, random_state=3), interpolation='bilinear')
axes[2].set_title("Positive Words")
axes[2].axis("off")
plt.tight_layout()
# plt.show();
return fig
def get_top_percent_words(doc, percent):
# Returns a list of "top-n" most frequent words in a list
top_n = int(percent * len(set(doc)))
counter = Counter(doc).most_common(top_n)
top_n_words = [x[0] for x in counter]
# print(top_n_words)
return top_n_words
def clean_document(doc):
spell = SpellChecker()
lemmatizer = WordNetLemmatizer()
# Lemmatize words (needed for calculating frequencies correctly )
doc = [lemmatizer.lemmatize(x) for x in doc]
# Get the top 10% of all words. This may include "misspelled" words
top_n_words = get_top_percent_words(doc, 0.1)
# Get a list of misspelled words
misspelled = spell.unknown(doc)
# Accept the correctly spelled words and top_n words
clean_words = [x for x in doc if x not in misspelled or x in top_n_words]
# Try to split the misspelled words to generate good words (ex. "lifeisstrange" -> ["life", "is", "strange"])
words_to_split = [x for x in doc if x in misspelled and x not in top_n_words]
split_words = flatten_list([wordninja.split(x) for x in words_to_split])
# Some splits may be nonsensical, so reject them ("llouis" -> ['ll', 'ou', "is"])
clean_words.extend(spell.known(split_words))
return clean_words
def get_log_likelihood(doc1, doc2):
doc1_counts = Counter(doc1)
doc1_freq = {
x: doc1_counts[x]/len(doc1)
for x in doc1_counts
}
doc2_counts = Counter(doc2)
doc2_freq = {
x: doc2_counts[x]/len(doc2)
for x in doc2_counts
}
doc_ratios = {
# 1 is added to prevent division by 0
x: math.log((doc1_freq[x] +1 )/(doc2_freq[x]+1))
for x in doc1_freq if x in doc2_freq
}
top_ratios = Counter(doc_ratios).most_common()
top_percent = int(0.1 * len(top_ratios))
return top_ratios[:top_percent]
# Function to generate a document based on likelihood values for words
def get_scaled_list(log_list):
counts = [int(x[1]*100000) for x in log_list]
words = [x[0] for x in log_list]
cloud = []
for i, word in enumerate(words):
cloud.extend([word]*counts[i])
# Shuffle to make it more "real"
random.shuffle(cloud)
return cloud
def get_smart_clouds(df):
neg_doc = flatten_list(df[df['sentiment_cat']=='negative']['words'])
neg_doc = [x for x in neg_doc if is_acceptable(x)]
pos_doc = flatten_list(df[df['sentiment_cat']=='positive']['words'])
pos_doc = [x for x in pos_doc if is_acceptable(x)]
neu_doc = flatten_list(df[df['sentiment_cat']=='neutral']['words'])
neu_doc = [x for x in neu_doc if is_acceptable(x)]
# Clean all the documents
neg_doc_clean = clean_document(neg_doc)
neu_doc_clean = clean_document(neu_doc)
pos_doc_clean = clean_document(pos_doc)
# Combine classes B and C to compare against A (ex. "positive" vs "non-positive")
top_neg_words = get_log_likelihood(neg_doc_clean, flatten_list([pos_doc_clean, neu_doc_clean]))
top_neu_words = get_log_likelihood(neu_doc_clean, flatten_list([pos_doc_clean, neg_doc_clean]))
top_pos_words = get_log_likelihood(pos_doc_clean, flatten_list([neu_doc_clean, neg_doc_clean]))
# Generate syntetic a corpus using our loglikelihood values
neg_doc_final = get_scaled_list(top_neg_words)
neu_doc_final = get_scaled_list(top_neu_words)
pos_doc_final = get_scaled_list(top_pos_words)
# Visualise our synthetic corpus
fig = generate_word_clouds(neg_doc_final, neu_doc_final, pos_doc_final)
return fig
source: google images
